# -*- coding: utf-8 -*-
'''
Created on Oct 30, 2010

@author: HoangNamHai
'''

import commonlib
import datetime
import os, traceback
from MongoModel import MongoModel
from MongoDbLog import MongoDbLog

LOG = MongoDbLog('ThoiTiet', 'debug')

MONGO_SERVER = 'beta.mana.vn'    # Server Mana 64bit
MONGO_PORT = 27017
DOWNLOAD_PATH = '/home/hoangnamhai/HarvestedData/tintuc/tym/thoitiet/'
PREFIX = '/uploads/thoitiet/'

class ThoiTiet(MongoModel):
    
    __urlThoiTietHienTai = 'http://www.nchmf.gov.vn/web/vi-VN/81/Default.aspx'
    __urlThoiTietDatLien = 'http://www.nchmf.gov.vn/web/vi-VN/79/12/Default.aspx'
    __urlThoiTietBien = 'http://www.nchmf.gov.vn/web/vi-VN/80/13/Default.aspx'
    
    def __init__(self, host='localhost', port=27017):
        MongoModel.__init__(self, host, port)
        
    def getThoitietHientai(self):
        LOG.debug('getThoiTietHienTai: %s' % self.__urlThoiTietHienTai)
        url = self.__urlThoiTietHienTai
        try:
            tree = commonlib.getXMLTree(url)
            data = []
            hasData = False
            for row in tree.xpath("//table[@class='tableLoop']//tr"):
                hasData = True
                dataRow = {}
                dataRow['ten'] = commonlib.getElementText(row.xpath('./td[1]'))
                dataRow['thoitiet'] = commonlib.getElementText(row.xpath('./td[2]'), descendant=1)
                dataRow['nhietdo'] = commonlib.getElementText(row.xpath('./td[3]//strong'))
                dataRow['doam'] = commonlib.getElementText(row.xpath('./td[4]'))
                dataRow['gio'] = commonlib.getElementText(row.xpath('./td[5]'))
                dataRow['icon'] = commonlib.getAttribText(row.xpath('./td[2]//img'), 'src', '')
                dataRow['icon2'] = commonlib.downloadImage(dataRow['icon'], DOWNLOAD_PATH, PREFIX)
                data.append(dataRow)
    #            for key in dataRow.keys():
    #                print '[{0}]: {1}'.format(key, dataRow[key].encode('utf-8'))
            if hasData == True:
                result = {}
                result['reporttime'] = commonlib.getElementText(tree.xpath("//td[@class='SubTitleNews_Special']"), descendant=1)
                result['_id'] = commonlib.getCRC32Unsign("thoitiethientai")
                result['data'] = data
                result['lastupdate'] = datetime.datetime.now()
                
                db = self.connection['thoitiet']    
                collection = db['hientai']    
                collection.save(result)
                print result
            else:
                LOG.warning('không tách được dữ liệu')
        except:
            LOG.error(traceback.format_exc())
            raise
            
    def xulyTextThoitiet(self, elem):
        result = ''
        for spanElem in elem.xpath('.//span[@class]'):
            temp = commonlib.getElementText(spanElem, descendant=1)
            temp = temp.expandtabs(0).strip()
            # temp = html2text.html2text(etree.tostring(spanElem))
            result = result + '\r\n' + temp
            result = result.replace(' từ : \r\n',' từ : ')
            result = result.replace('\r\noC','oC')
        return result
    
    def getThoitietDatlien(self):
        LOG.info('start getThoiTietDatLien: %s' % self.__urlThoiTietDatLien)
        try:
            url = self.__urlThoiTietDatLien
            tree = commonlib.getXMLTree(url)
            data = []
            for row in tree.xpath("//table[@class='dl_Bantindubao']"):
                dataRow = {}
                dataRow['ten'] = commonlib.getElementText(row.xpath('.//td[@class="header_vung"]'), descendant=1)
                dataRow['thoitiet'] = self.xulyTextThoitiet(row)
                dataRow['icon'] = commonlib.getAttribText(row.xpath('.//td[2]/img'), 'src', '')
                dataRow['icon2'] = commonlib.downloadImage(dataRow['icon'], DOWNLOAD_PATH, PREFIX)
                data.append(dataRow)
    #            for key in dataRow.keys():
    #                print '[{0}]: {1}'.format(key, dataRow[key].encode('utf-8'))
            result = {}
            result['reporttime'] = commonlib.getElementText(tree.xpath("//*[@class='SubTitleNews_Special']"), descendant=1)
            result['_id'] = commonlib.getCRC32Unsign("thoitietdatlien")
            result['data'] = data
            result['lastupdate'] = datetime.datetime.now()
            
            db = self.connection['thoitiet']
            collection = db['datlien']    
            collection.save(result)
            print result['reporttime']
            print result
        except:
            LOG.error(traceback.format_exc())
            raise
            
    def getThoitietBien(self):
        LOG.debug('start getThoiTietBien: %s' % self.__urlThoiTietBien)
        try:
            url = self.__urlThoiTietBien
            tree = commonlib.getXMLTree(url)
            data = []
            for row in tree.xpath("//table[@class='dl_Bantindubao']"):
                dataRow = {}
                dataRow['ten'] = commonlib.getElementText(row.xpath('.//td[@class="header_vung"]'), descendant=1)
                dataRow['thoitiet'] = self.xulyTextThoitiet(row)
                data.append(dataRow)
    #            for key in dataRow.keys():
    #                print '[{0}]: {1}'.format(key, dataRow[key].encode('utf-8'))
            result = {}
            result['reporttime'] = commonlib.getElementText(tree.xpath("//*[@class='SubTitleNews_Special']"), descendant=1)
            result['_id'] = commonlib.getCRC32Unsign("thoitietbien")
            result['data'] = data
            result['lastupdate'] = str(datetime.datetime.now())
            
            db = self.connection['thoitiet']
            collection = db['bien']    
            collection.save(result)
            print result['reporttime']
            print result
        except:
            LOG.error(traceback.format_exc())
            raise


if __name__ == '__main__':
    LOG.info('start crawler thoi tiet tren trang "%s"' % 'www.nchmf.gov.vn')   
    try: 
        weather = ThoiTiet(MONGO_SERVER, MONGO_PORT)
#        weather = ThoiTiet()
        weather.getThoitietDatlien()
        weather.getThoitietBien()
        weather.getThoitietHientai()  
    except:
        LOG.error(traceback.format_exc())
    LOG.info('finished crawler thoi tiet luc: {0}'.format(datetime.datetime.now()))
    pid = os.getpid()
    os._exit(1)
    os.kill(pid, 9)
    